# -*- coding: UTF-8 -*-
import requests
import pymysql
import sys
import re
import hashlib
import os
from lxml import etree
import time
from collections import OrderedDict
import json

# 代码写的像shi一样, 回头重写
# 2018-08-07 重写代码,移步旁边mahle_for_pid_again

# 解决编码问题
reload(sys)
sys.setdefaultencoding('utf-8')

class Mahle_For_Pid(object):

    def __init__(self, db_config):
        self.headers = {
            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
            "Accept-Encoding": "gzip, deflate",
            "Accept-Language": "zh-CN,zh;q=0.9",
            "Cache-Control": "max-age=0",
            "Connection": "keep-alive",
            "Content-Type": "application/x-www-form-urlencoded",
            "Host": "catalog.mahle-aftermarket.cn",
            "Origin": "http://catalog.mahle-aftermarket.cn",
            "Referer": "http://catalog.mahle-aftermarket.cn/cn/modules/oeFilter/index.xhtml",
            "Upgrade-Insecure-Requests": "1",
            "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36",
            'Cookie': '__utmz=94631182.1531893131.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); __utmc=94631182; JSESSIONID=10270BD9FC17F491E6893D6C3C9D2D14; __utma=94631182.70392564.1531893131.1532598267.1533003241.7; __utmb=94631182.0.10.1533003241',}
        self.last_headers = {
            "Accept": "*/*",
            "Accept-Encoding": "gzip, deflate",
            "Accept-Language": "zh-CN,zh;q=0.9",
            "Cache-Control": "no-cache",
            "Connection": "keep-alive",
            "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
            "Cookie": "__utmz=94631182.1531893131.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); __utmc=94631182; JSESSIONID=10270BD9FC17F491E6893D6C3C9D2D14; __utma=94631182.70392564.1531893131.1532598267.1533003241.7; __utmb=94631182.0.10.1533003241",
            "Host": "catalog.mahle-aftermarket.cn",
            "Origin": "http://catalog.mahle-aftermarket.cn",
            "Pragma": "no-cache",
            "Referer": "",
            "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36",
            "Proxy-Connection": "keep-alive",
        }

        self.search_post_data = OrderedDict()  # 搜索发送的data
        self.search_post_data['autoScroll']= '',
        self.search_post_data['statistic_ftsInputFeld']= '',
        self.search_post_data['validationFtsModalBoxOpenedState']= '',
        self.search_post_data['waitModalBoxOpenedState']= '',
        self.search_post_data['statistic_bezeichnung']= '',
        self.search_post_data['j_id13_SUBMIT']= '1',
        self.search_post_data['javax.faces.ViewState']= '',
        self.search_post_data['j_id13:_idcl']= 'searchButton',

        self.last_post_data = OrderedDict()
        self.last_post_data['AJAXREQUEST'] = ''
        self.last_post_data['autoScroll'] = ''
        self.last_post_data['statistic_ftsInputFeld'] = ''
        self.last_post_data['validationFtsModalBoxOpenedState'] = ''
        self.last_post_data['waitModalBoxOpenedState'] = ''
        self.last_post_data['j_id13_SUBMIT'] = '1'
        self.last_post_data['javax.faces.ViewState'] = ''
        self.last_post_data['hersteller'] = ''
        self.last_post_data['filterId'] = ''
        self.last_post_data['show'] = 'oeFilter'
        self.last_post_data['ajaxSingle'] = 'hersteller'


        # self.search_post_data = {
        #     'autoScroll':'',
        #     'statistic_ftsInputFeld':'',
        #     'validationFtsModalBoxOpenedState':'',
        #     'waitModalBoxOpenedState':'',
        #     'statistic_bezeichnung':'',
        #     'j_id13_SUBMIT': '1',
        #     'javax.faces.ViewState': '',
        #     'j_id13:_idcl': 'searchButton',
        # }
        # self.last_post_data = {
        #     'AJAXREQUEST': '',
        #     'autoScroll': '',
        #     'statistic_ftsInputFeld': '',
        #     'validationFtsModalBoxOpenedState': '',
        #     'waitModalBoxOpenedState': '',
        #     'j_id13_SUBMIT': '1',
        #     'javax.faces.ViewState': '',
        #     'hersteller': '',
        #     'filterId': '',
        #     'show': 'oeFilter',
        #     'ajaxSingle': 'hersteller',
        # }
        self.start_url = "http://catalog.mahle-aftermarket.cn/cn/modules/oeFilter/index.xhtml"
        self.search_url = "http://catalog.mahle-aftermarket.cn/cn/modules/oeFilter/index.xhtml"
        self.base_path = "/Users/duanmengdi/Desktop/workspace/mahle_html/"
        self.javax_value = ""
        self.db_config = db_config
        self.s = requests.session()
        self.total_num = 0
        self.conn = None
        self.curs = None
        self.conn2 = None
        self.curs2 = None

    def get_cursor(self, type='stream'):
        self.conn = pymysql.connect(**self.db_config)
        if type == 'stream':
            return self.conn.cursor(pymysql.cursors.SSCursor)  # 返回流式游标,查询大量数据时不占用内存(返回数据形式是元组)
        elif type == 'dict':
            return self.conn.cursor(pymysql.cursors.DictCursor)  # 返回字典形式游标,查询出的数据以字典形式返回
        else:
            raise Exception("cursor type error")

    def get_cursor2(self, type='stream'):
        self.conn2 = pymysql.connect(**self.db_config)
        if type == 'stream':
            return self.conn2.cursor(pymysql.cursors.SSCursor)  # 返回流式游标,查询大量数据时不占用内存(返回数据形式是元组)
        elif type == 'dict':
            return self.conn2.cursor(pymysql.cursors.DictCursor)  # 返回字典形式游标,查询出的数据以字典形式返回
        else:
            raise Exception("cursor type error")

    def getmd5(self, str):
        md5 = hashlib.md5()
        md5.update(str)
        return md5.hexdigest()

    def parse_url(self, url, method, post_data=None, headers=None):
        if method == 'GET':
            data = "GET" + url  # 拼接data
            md5 = self.getmd5(data)  # 获取此次请求电子指纹
            file_path = self.base_path + md5 + ".html"
            if os.path.exists(file_path):  # 判断文件是否存在,如果存在就从本地读取
                print "GET for local"
                with open(file_path, 'r') as f:
                    res = f.read()
                response = res
            else:  # 如果路径不存在, 从web端请求
                print "GET fro web"
                try:
                    response = requests.get(url=url, verify=False, headers=self.headers, timeout=30, allow_redirects=False)
                except Exception as e:
                    response = requests.get(url=url, verify=False, headers=self.headers, timeout=30, allow_redirects=False)

                if response:
                    with open(file_path, 'w') as f:  # 保存此次请求的html
                        f.write(response.content.decode('utf8'))
                    response = response.content.decode("utf8")
                else:
                    response = None

        elif method == "POST":
            # 之前旧的创建MD5方法,不严谨 data = "POST" + url + post_data.get("statistic_bezeichnung", "0") + post_data.get("AJAXREQUEST", "0") + post_data.get("hersteller", "0") + post_data.get("filterId", "0")
            data = "POST" + url + json.dumps(post_data)
            md5 = self.getmd5(data)
            file_path = self.base_path + md5 + ".html"
            if os.path.exists(file_path):  # 判断文件是否存在,如果存在就从本地读取
                print "POST for local"
                with open(file_path, 'r') as f:
                    res = f.read()
                response = res
            else:
                print "POST for web"
                try:
                    response = requests.post(url=url, data=post_data, verify=False, headers=headers, timeout=30, allow_redirects=False)
                except Exception as e:
                    response = requests.post(url=url, data=post_data, verify=False, headers=headers, timeout=30, allow_redirects=False)
                if response:
                    # self.cookies = response.cookies.get_dict()
                    # print "______"
                    # print self.cookies
                    # print "______"

                    with open(file_path, 'w') as f:  # 保存此次请求的html
                        f.write(response.content.decode('utf8'))
                    response = response.content.decode("utf8")
                else:
                    response = None

        return response

    def save2db(self, dict):

        pid = dict['pid'].decode('utf8')
        brand_name = dict['brand_name'].decode('utf8')
        part_brand = dict['part_brand'].decode('utf8')
        part_num = dict['part_num'].decode('utf8')
        part_name = dict['part_name'].decode('utf8')
        brand_name1 = dict['brand_name1'].decode('utf8')
        pid1 = dict['pid1'].decode('utf8')
        sql = '''INSERT INTO `mahle_all` (`pid`, `brand_name`, `part_brand`, `part_num`, `part_name`, `brand_name1`, `pid1`) VALUES ('%s', '%s', '%s', '%s', '%s', '%s', '%s')''' % (pid, brand_name, part_brand, part_num, part_name, brand_name1, pid1)
        try:
            self.curs2.execute(sql)
        except Exception as e:
            print e
            print sql
            exit()
        # self.conn.commit()
        self.total_num += 1

    def run(self):
        self.curs2 = self.get_cursor2()
        # 从数据库中读取pid
        self.curs = self.get_cursor()
        sql = '''select brand,item,pid from 0731_pid'''  # 根据数据源的不同需要修改
        self.curs.execute(sql)
        for data in self.curs:  # 遍历游标,获取每一条数据,搜索每一条pid
            time.sleep(1)  # 从本地跑数据的时候,可以注释掉
            local_pid = data[2].replace('.0', '')  # 当前条数据的pid
            brand_name = data[0]
            # 访问初始入口url, 如果有就从本地请求
            #response = self.parse_url(url=self.start_url, method="GET")
            # 先访问初始入口url, 每次都从web端请求
            response = requests.get(url=self.start_url, verify=False, headers=self.headers, timeout=30, allow_redirects=False)
            # 获取隐藏域参数并更新headers
            search_html = etree.HTML(response.content.decode('utf8'))
            self.javax_value = search_html.xpath("//input[@id='javax.faces.ViewState']/@value")
            self.javax_value = self.javax_value[0] if len(self.javax_value)>0 else ""
            # print  self.javax_value
            # 更新headers
            self.search_post_data['javax.faces.ViewState'] = self.javax_value

            # 搜索入口页面获取完毕,开始发送post请求
            # 拼装search_post_data
            self.search_post_data['statistic_bezeichnung'] = local_pid
            print "提交搜索", self.search_post_data
            response = self.parse_url(url=self.search_url, method="POST", post_data=self.search_post_data, headers=self.headers)
            search_result_html = etree.HTML(response)  # 获取搜索结果页面HTML对象

            detail_url_list = search_result_html.xpath("//a[@class='detailLink']/@href")  # 获取详情页面url

            if len(detail_url_list) <= 0:
                print "没有找到详情url, 跳过", local_pid
            for detail_url in detail_url_list:

                # 拼装详情页url, 并请求,获取part_num, part_name
                detail_url = 'http://catalog.mahle-aftermarket.cn' + detail_url
                print "详情url", detail_url
                response = self.parse_url(url=detail_url, method="GET")
                detail_html = etree.HTML(response)
                part_data = detail_html.xpath("//td[@class='data'][1]/input/@value")
                oefilter_url = detail_url + "&show=oeFilter"  # 拼接进入OE型号标签界面的url
                fahrzeug_url = detail_url + "&show=fahrzeug"  # 拼接进入车辆标签界面的url

                self.parse_url(url=fahrzeug_url, method="GET")  # 该界面仅仅做保存,为以后使用

                part_num = part_data[0]
                part_name = part_data[1]

                # 进入oe型号详情页, get请求
                oe_response = self.parse_url(url=oefilter_url, method="GET")
                oe_html = etree.HTML(oe_response)

                #####
                # 先获取显示出来的数据
                one_tr_list = oe_html.xpath("//tbody[contains(@id,':tbody_element')]/tr")
                for tr in one_tr_list:
                    brand_name1 = tr.xpath("./td[1]/text()")
                    pid1 = tr.xpath("./td[2]/text()")
                    brand_name1 = brand_name1[0] if len(brand_name1) > 0 else ""
                    pid1 = pid1[0] if len(pid1) > 0 else ""
                    if pid1:
                        pid1 = pid1.replace(" ", '')

                    last_data = dict(
                        pid=local_pid,
                        brand_name=brand_name,
                        part_brand='mahle',  # mahle的网站,当然都是mahle
                        part_num=part_num.replace(' ', ''),
                        part_name=part_name,
                        brand_name1=brand_name1.replace(' ', ''),
                        pid1=pid1
                    )
                    # if pid1 == local_pid:
                    #     print "pid1==local_pid", pid1, local_pid
                    #     continue
                    self.save2db(last_data)
                    print "___", last_data
                    # exit()
                #####



                last_post_url = "http://catalog.mahle-aftermarket.cn/cn/modules/mahleFilter/detail.xhtml"  # 最终的post提交url

                # 获取一些必须得参数
                select = oe_html.xpath("//select[@id='hersteller']")
                select = select[0] if len(select)>0 else None
                if select is None:
                    print "没有找到select下拉菜单, 跳过", local_pid
                    continue
                onchange = select.xpath("./@onchange")
                onchange = onchange[0] if len(onchange)>0 else None
                if onchange is None:
                    print "没有找到包含的必要参数, 跳过", local_pid
                    continue
                print "最后一次post请求所必要的参数", onchange

                # 获取一些必要的参数(都是请求里需要的参数)
                AJAXREQUEST = re.findall("'containerId':'(.*?)'", onchange)
                AJAXREQUEST = AJAXREQUEST[0] if len(AJAXREQUEST)>0 else ""
                filterId = re.findall("'filterId':'(.*?)'", onchange)
                filterId = filterId[0] if len(filterId)>0 else ""
                jid = re.findall("j_id.*?(j_id.*?)'", onchange)
                jid = jid[0] if len(jid)>0 else ""


                self.javax_value = oe_html.xpath("//input[@id='javax.faces.ViewState']/@value")
                self.javax_value = self.javax_value[0] if len(self.javax_value)>0 else ""

                option_list = select.xpath("./option")  # 获取option列表(车型下拉菜单)
                # option_list.pop(0)
                for option in option_list:
                    brand_name1 = option.xpath("./text()")
                    brand_name1 = brand_name1[0] if len(brand_name1)>0 else None
                    hersteller = option.xpath("./@value")
                    hersteller = hersteller[0] if len(hersteller)>0 else None
                    if brand_name1 is None or hersteller is None:
                        print "没有找到brand_name1或者对应的value", local_pid
                        continue

                    # 拼装最终的post请求
                    self.last_post_data['AJAXREQUEST'] = AJAXREQUEST
                    self.last_post_data['filterId'] = filterId
                    self.last_post_data[jid] = jid
                    self.last_post_data['hersteller'] = hersteller
                    self.last_post_data['javax.faces.ViewState'] = self.javax_value

                    # 更新last_headers里的referer
                    self.last_headers['Referer'] = oefilter_url
                    # print self.last_post_data  # 打印最后一次请求发送的数据
                    # print self.last_headers  # 打印最后一次请求的headers头
                    # time.sleep(5)
                    last_response = self.parse_url(url=last_post_url, method="POST", post_data=self.last_post_data, headers=self.last_headers)
                    # print "----"
                    # print last_response

                    # 踩过的坑,保留纪念, javax_value应该从当前页面获取, 因为上边写过获取这个参数的代码,就复制下来直接用了,没有改动,
                    # 所以拿错数据,导致请求不成功, 以后复制代码小心小心小心再小心
                    # content = re.findall('"Location" content="(.*?)"', last_response)[0]
                    # print content
                    # test_last_url = "http://catalog.mahle-aftermarket.cn" + content
                    # print test_last_url
                    # response = self.parse_url(url=test_last_url, method="POST", post_data=self.last_post_data, headers=self.last_headers)
                    # response2 = self.parse_url(url=test_last_url, method="GET", headers=self.last_headers)


                    #
                    try:
                        last_html = etree.HTML(last_response)
                    except ValueError:  # 如果报valueerror, 就是javaxvalue从本地读取了,那么从web端再请求一次,
                        oe_response = requests.get(url=oefilter_url, verify=False, headers=self.headers, timeout=30, allow_redirects=False)
                        oe_html = etree.HTML(oe_response.content.decode('utf8'))

                        #####
                        # 先获取显示出来的数据
                        one_tr_list = oe_html.xpath("//tbody[contains(@id,':tbody_element')]/tr")
                        for tr in one_tr_list:
                            brand_name1 = tr.xpath("./td[1]/text()")
                            pid1 = tr.xpath("./td[2]/text()")
                            brand_name1 = brand_name1[0] if len(brand_name1) > 0 else ""
                            pid1 = pid1[0] if len(pid1) > 0 else ""
                            if pid1:
                                pid1 = pid1.replace(" ", '')

                            last_data = dict(
                                pid=local_pid,
                                brand_name=brand_name,
                                part_brand='mahle',  # mahle的网站,当然都是mahle
                                part_num=part_num.replace(' ', ''),
                                part_name=part_name,
                                brand_name1=brand_name1.replace(' ', ''),
                                pid1=pid1
                            )
                            # if pid1 == local_pid:
                            #     print "pid1==local_pid", pid1, local_pid
                            #     continue
                            self.save2db(last_data)
                            print "___", last_data
                            # exit()
                            #####


                        # 获取一些必须得参数
                        select = oe_html.xpath("//select[@id='hersteller']")
                        select = select[0] if len(select) > 0 else None
                        if select is None:
                            print "没有找到select下拉菜单, 跳过", local_pid
                            continue
                        onchange = select.xpath("./@onchange")
                        onchange = onchange[0] if len(onchange) > 0 else None
                        if onchange is None:
                            print "没有找到包含的必要参数, 跳过", local_pid
                            continue
                        print "最后一次post请求所必要的参数", onchange

                        # 获取一些必要的参数(都是请求里需要的参数)
                        AJAXREQUEST = re.findall("'containerId':'(.*?)'", onchange)
                        AJAXREQUEST = AJAXREQUEST[0] if len(AJAXREQUEST) > 0 else ""
                        filterId = re.findall("'filterId':'(.*?)'", onchange)
                        filterId = filterId[0] if len(filterId) > 0 else ""
                        jid = re.findall("j_id.*?(j_id.*?)'", onchange)
                        jid = jid[0] if len(jid) > 0 else ""

                        self.javax_value = oe_html.xpath("//input[@id='javax.faces.ViewState']/@value")
                        self.javax_value = self.javax_value[0] if len(self.javax_value) > 0 else ""

                        option_list = select.xpath("./option")  # 获取option列表(车型下拉菜单)
                        option_list.pop(0)
                        for option in option_list:
                            brand_name1 = option.xpath("./text()")
                            brand_name1 = brand_name1[0] if len(brand_name1) > 0 else None
                            hersteller = option.xpath("./@value")
                            hersteller = hersteller[0] if len(hersteller) > 0 else None
                            if brand_name1 is None or hersteller is None:
                                print "没有找到brand_name1或者对应的value", local_pid
                                continue

                            # 拼装最终的post请求
                            self.last_post_data['AJAXREQUEST'] = AJAXREQUEST
                            self.last_post_data['filterId'] = filterId
                            self.last_post_data[jid] = jid
                            self.last_post_data['hersteller'] = hersteller
                            self.last_post_data['javax.faces.ViewState'] = self.javax_value

                            # 更新last_headers里的referer
                            self.last_headers['Referer'] = oefilter_url
                            # print self.last_post_data  # 打印最后一次请求发送的数据
                            # print self.last_headers  # 打印最后一次请求的headers头
                            # time.sleep(5)
                            last_response = self.parse_url(url=last_post_url, method="POST",post_data=self.last_post_data, headers=self.last_headers)
                            last_html = etree.HTML(last_response)
                            tr_list = last_html.xpath("//tbody[contains(@id,':tbody_element')]/tr")
                            for tr in tr_list:
                                brand_name1 = tr.xpath("./td[1]/text()")
                                pid1 = tr.xpath("./td[2]/text()")
                                brand_name1 = brand_name1[0] if len(brand_name1) > 0 else ""
                                pid1 = pid1[0] if len(pid1) > 0 else ""
                                if pid1:
                                    pid1 = pid1.replace(" ", '')

                                last_data = dict(
                                    pid=local_pid,
                                    brand_name=brand_name,
                                    part_brand='mahle',  # mahle的网站,当然都是mahle
                                    part_num=part_num.replace(' ', ''),
                                    part_name=part_name,
                                    brand_name1=brand_name1.replace(' ', ''),
                                    pid1=pid1
                                )
                                # if pid1 == local_pid:
                                #     print "pid1==local_pid", pid1, local_pid
                                #     continue
                                print "___", last_data
                                self.save2db(last_data)


                    tr_list = last_html.xpath("//tbody[contains(@id,':tbody_element')]/tr")
                    for tr in tr_list:
                        brand_name1 = tr.xpath("./td[1]/text()")
                        pid1 = tr.xpath("./td[2]/text()")
                        brand_name1 = brand_name1[0] if len(brand_name1)>0 else ""
                        pid1 = pid1[0] if len(pid1)>0 else ""
                        if pid1:
                            pid1 = pid1.replace(" ", '')

                        last_data = dict(
                            pid=local_pid,
                            brand_name=brand_name,
                            part_brand='mahle',  # mahle的网站,当然都是mahle
                            part_num=part_num.replace(' ', ''),
                            part_name=part_name,
                            brand_name1=brand_name1.replace(' ', ''),
                            pid1=pid1
                        )
                        # if pid1 == local_pid:
                        #     print "pid1==local_pid", pid1, local_pid
                        #     continue
                        print "___", last_data
                        self.save2db(last_data)

        self.conn2.commit()
        print self.total_num


if __name__ == '__main__':
    db_config = dict(
        host = "127.0.0.1",
        port = 3306,
        user = "root",
        passwd = "123456",
        db = "my_data",
        charset = "utf8")

    tools = Mahle_For_Pid(db_config=db_config)
    tools.run()